/* Complet (Autocompletion through Graph Databases)
 *  Copyright (C) 2012  Paul Wagner, Till Speicher
 *
 *  This program is free software; you can redistribute it and/or modify it
 *  under the terms of the GNU General Public License as published by the
 *  Free Software Foundation; either version 3 of the License, or (at your
 *  option) any later version.
 *
 *  This program is distributed in the hope that it will be useful, but
 *  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 *  or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
 *  for more details.
 *
 *  You should have received a copy of the GNU General Public License along
 *  with this program; if not, see <http:/* www.gnu.org/licenses/>.
 */

/*
 * Main standalone DataCollector with embedded Database.
 * This class uses the GraphDatabaseService to access Neo4j databases and insert google n-grams data.
 *
 * @version 2.1
 * @author Paul Wagner, René Pickhardt
 *
 */
package complet.app.io.datacollector;

import complet.db.DBConn;
import complet.helper.ConfigHelper;
import java.io.BufferedReader;
import java.io.IOException;
import java.util.HashMap;
import complet.helper.IOHelper;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.TreeMap;
import org.neo4j.graphdb.Direction;

import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Relationship;

public class DataCollector {

    private static int anz_stnce = -1; // -1 means complete file
    private static TreeMap<Integer, String> map_stnce = new TreeMap<Integer, String>();
    //private static ArrayList<String> map_stnce = new ArrayList<String>();
    private static HashMap<String, Node> nodeMap = new HashMap<String, Node>();
    private static HashMap<String, Relationship> relMap = new HashMap<String, Relationship>();
    private static DBConn conn = null;

    public static void main(String[] args) {

        ConfigHelper.IN_PATH = "E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\inputfiles\\ngrams\\5grams_uc.txt";
        ConfigHelper.DB_PATH = "E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\dbs\\testtodekete.db";
        ConfigHelper.BUFFER = false;

        if (args.length > 0) {
            ConfigHelper.loadConfigFile(args[0]);
        }

        // Print config values
        IOHelper.pln("##### CONFIG Values #####");
        IOHelper.pln("NAME_KEY: " + ConfigHelper.NAME_KEY);
        IOHelper.pln("COUNT_KEY: " + ConfigHelper.COUNT_KEY);
        IOHelper.pln("MAX_RELATIONS: " + ConfigHelper.MAX_RELATIONS);
        IOHelper.pln("");
        IOHelper.pln("##### Start DB Operations #####");
        IOHelper.pln("Using DB: " + ConfigHelper.DB_PATH);

        // Buffer sentences
        BufferedReader in = IOHelper.openReadFile(ConfigHelper.IN_PATH);
        String strLine;
        int c = 1;

        // Establish DB connection
        conn = new DBConn();
        conn.openTransaction();

        // Buffer database
        IOHelper.pln("Buffer all relationships");

        int cnt = 0;
        String sn1 = "";
        String sn2 = "";
        String relMapKey = "";
        for (Node n1 : conn.getGraph().getAllNodes()) {
            cnt++;
            if (!n1.hasProperty(ConfigHelper.NAME_KEY)) {
                continue;
            }
            sn1 = (String) n1.getProperty(ConfigHelper.NAME_KEY);
            nodeMap.put(sn1, n1);
            for (Relationship r : n1.getRelationships(Direction.OUTGOING)) {
                sn2 = (String) r.getEndNode().getProperty(ConfigHelper.NAME_KEY);
                relMapKey = sn1 + ":" + sn2 + ":" + r.getType().name();
                relMap.put(relMapKey, r);
            }
            for (Relationship r : n1.getRelationships(conn.getDn()[1], Direction.OUTGOING)) {
                sn2 = (String) r.getEndNode().getProperty(ConfigHelper.NAME_KEY);
                relMapKey = sn1 + ":" + sn2 + ":1";
                relMap.put(relMapKey, r);
            }
            for (Relationship r : n1.getRelationships(conn.getDn()[2], Direction.OUTGOING)) {
                sn2 = (String) r.getEndNode().getProperty(ConfigHelper.NAME_KEY);
                relMapKey = sn1 + ":" + sn2 + ":2";
                relMap.put(relMapKey, r);
            }
            for (Relationship r : n1.getRelationships(conn.getDn()[3], Direction.OUTGOING)) {
                sn2 = (String) r.getEndNode().getProperty(ConfigHelper.NAME_KEY);
                relMapKey = sn1 + ":" + sn2 + ":3";
                relMap.put(relMapKey, r);
            }
            for (Relationship r : n1.getRelationships(conn.getDn()[4], Direction.OUTGOING)) {
                sn2 = (String) r.getEndNode().getProperty(ConfigHelper.NAME_KEY);
                relMapKey = sn1 + ":" + sn2 + ":4";
                relMap.put(relMapKey, r);
            }
            IOHelper.pln(cnt + ": Next node...");
        }

        String word = "";
        cnt = 0;
        Boolean next_stnce = false;
        try {

//            int my_cnt = 0;
//            while (true && (strLine = in.readLine()) != null) {
//                my_cnt++;
//                if (my_cnt > 40599999) {
//                    break;
//                }
//            }

            while (true && (strLine = in.readLine()) != null) {
                next_stnce = true;

//                my_cnt++;
//                if (my_cnt >= 40599999) {
//                    break;
//                }

                if (false) {
                    // WIKIPEDIA
                    String[] values = strLine.split("\t");                 // Wikipedia Dataset is already lowercased and trimmed, too bad
                    if (values.length == 0) {
                        continue;
                    }
                    int occ = 1;
                    values[0] = values[0].replaceAll("[^a-zA-ZßüäöÜÄÖ ]", "");
                    String[] words = values[0].split(" ");

                    // String in ngram-steps durchgehen
                    // Bei 1-grams nodes hinzufügen, falls noch nicht existent

                    cnt++;
                    if (cnt % 10000 == 9999) {
                        IOHelper.pln("Commiting at " + cnt);
                        conn.commitTransaction();
                    }
                    if (cnt > 400000) {
                        break;
                    }

                    for (int s = 1; s < ConfigHelper.MAX_RELATIONS + 1; s++) {
                        int i = 0;
                        while (i < words.length) {
                            // Wenn Step 1, dann Nodes hinzufügen
                            if (s == 1) {
                                word = words[i];
                                if (nodeMap.get(word) != null) {
                                    Node n = nodeMap.get(word);
                                    Integer n_cnt = (Integer) nodeMap.get(word).getProperty(ConfigHelper.COUNT_KEY);
                                    n.setProperty(ConfigHelper.COUNT_KEY, n_cnt + 1);
                                } else {
                                    Node n = conn.getGraph().createNode();
                                    n.setProperty(ConfigHelper.NAME_KEY, word);
                                    n.setProperty(ConfigHelper.COUNT_KEY, 1);
                                    nodeMap.put(word, n);
                                }
                            }
                            // Wenn i>1, Beziehung von i-s zu i mit typ s hinzufügen
                            if (i > 0) {
                                String myrelMapKey = words[i - s] + ":" + words[i] + ":" + s;
                                Relationship rel = relMap.get(myrelMapKey);
                                if (rel != null) {
                                    Integer mycnt = (Integer) rel.getProperty(ConfigHelper.COUNT_KEY);
                                    rel.setProperty(ConfigHelper.COUNT_KEY, mycnt + occ);
                                } else {
                                    rel = nodeMap.get(words[i - s]).createRelationshipTo(nodeMap.get(words[i]), conn.getDn()[s]);
                                    rel.setProperty(ConfigHelper.COUNT_KEY, occ);
                                    relMap.put(relMapKey, rel);
                                }
                            }
                            i += s;
                        }
                    }
                } else {
                    // NGRAMS
                    //String[] values = strLine.split("\t");                 // NO LOWERCASE
                    String[] values = strLine.toLowerCase().split("\t");     // LOWERCASE
                    int occ = 1;
                    if (values.length > 1) {
                        occ = Integer.parseInt(values[1]);
                    }
                    values[0] = values[0].replaceAll("[^a-zA-ZßüäöÜÄÖ, ]", "");  // DEUTSCH
                    //values[0] = values[0].replaceAll("[^a-zA-ZßüäöÜÄÖ', ]", "");  // ENGLISH

                    String[] words = values[0].split(" ");
                    if (words.length < 5) {
                        continue;
                    }

                    //while(words.length>0 && words[0].startsWith("'"))             // ALSO ENGLISH
                    //    words = Arrays.copyOfRange(words, 1, words.length);

                    String[] current = new String[5];

                    if (false) {
                        // Zusammengesetzte Wörter
                        String s = "";
                        for (int x = 0; x < words.length; x++) {
                            if (words[x].charAt(0) >= 65 && words[x].charAt(0) <= 90) {
                                s += words[x] + " ";
                                continue;
                            }
                            if (words[x].length() <= 3 && x + 1 < words.length) {
                                s += words[x] + ";" + words[x + 1] + " ";
                                x++;
                            } else {
                                s += words[x] + " ";
                            }
                        }
                        words = s.trim().split(" ");
                        for (int x = 0; x < words.length; x++) {
                            words[x] = words[x].replace(';', ' ');
                        }
                    }

                    int h = words.length - 4;
                    if (words.length > 1) {
                        if (words.length <= 5) {
                            h = 1;
                        }
                        for (int j = 0; j < h; j++) {
                            if (words.length > 5) {
                                System.arraycopy(words, j, current, 0, 5);
                            } else {
                                current = words;
                            }
                            for (int i = 0; i < current.length && i < ConfigHelper.MAX_RELATIONS + 1; i++) {
                                cnt++;

                                if (cnt % 50000 == 49999) {
                                    IOHelper.pln("Commiting at " + cnt);
                                    conn.commitTransaction();
                                }
                                word = current[i];
                                if (nodeMap.get(word) != null) {
                                    if (next_stnce) {
                                        Node n = nodeMap.get(word);
                                        Integer n_cnt = (Integer) nodeMap.get(word).getProperty(ConfigHelper.COUNT_KEY);
                                        n.setProperty(ConfigHelper.COUNT_KEY, n_cnt + 1);
                                    }
                                    continue;
                                }
                                Node n = conn.getGraph().createNode();
                                n.setProperty(ConfigHelper.NAME_KEY, word);
                                n.setProperty(ConfigHelper.COUNT_KEY, 1);
                                nodeMap.put(word, n);
                            }

                            Node[] wordNodes = new Node[words.length];
                            boolean chk = true;

                            for (int i = 0; i < words.length && i < ConfigHelper.MAX_RELATIONS + 1; i++) {
                                wordNodes[i] = nodeMap.get(words[i]);

                                if (wordNodes[i] == null) {
                                    chk = false;
                                }
                            }

                            if(chk){
                                for (int i = 1; i < current.length && i < ConfigHelper.MAX_RELATIONS + 1; i++) {
                                    nSuccessor(current, wordNodes, i, occ);
                                }
                            }
                            next_stnce = false;
                        }
                    }
                }

                conn.getTx().success();
            }



            conn.getGraph().getReferenceNode().setProperty(ConfigHelper.NAME_KEY, "REF");
            conn.commitTransaction();
            IOHelper.pln("INSERT FINISHED! (" + nodeMap.size() + " nodes)");



            for (Node n : conn.getGraph().getAllNodes()) {
                if (n.hasProperty(ConfigHelper.NAME_KEY)) {
                    word = (String) n.getProperty(ConfigHelper.NAME_KEY);


                    if (conn.getWordIndex().get(ConfigHelper.NAME_KEY, word).size() == 0) {
                        conn.getWordIndex().add(n, ConfigHelper.NAME_KEY, word);


                    } //IOHelper.log(word);
                }
            }

        } catch (Exception e) {
            IOHelper.logError("(DataCollector) Failed to insert input file, " + e.getMessage());


        } finally {
            conn.closeTransaction();


        }

        conn.closeConnection();


    }

    private static void nSuccessor(String[] words, Node[] wordNodes, int n, int occurences) {
        int len = words.length;

        for (int i = 0; i + n < len && i < ConfigHelper.MAX_RELATIONS + 1; i++) {
            String relMapKey = words[i] + ":" + words[i + n] + ":" + n;
            Relationship rel = relMap.get(relMapKey);

            if (rel != null) {
                Integer cnt = (Integer) rel.getProperty(ConfigHelper.COUNT_KEY);
                rel.setProperty(ConfigHelper.COUNT_KEY, cnt + occurences);


            } else {
                rel = wordNodes[i].createRelationshipTo(wordNodes[i + n], conn.getDn()[n]);
                rel.setProperty(ConfigHelper.COUNT_KEY, occurences);
                relMap.put(relMapKey, rel);

            }
        }
    }
}
